static inline unsigned long pmd_val(pmd_t x)
{
unsigned long ret = x.pmd;
- if ( (ret & 1) ) ret = machine_to_phys(ret);
+ if (!(ret & 0x801) && ret) printk("pmd_val really invalid!!!\n");
+ if (ret) ret = machine_to_phys(ret);
return ret;
}
#define pgd_val(x) ({ BUG(); (unsigned long)0; })
{
clear_page(pte);
__make_page_readonly(pte);
- queue_pte_pin(__pa(pte));
+ // queue_pte_pin(__pa(pte));
}
return pte;
static __inline__ void pte_free_slow(pte_t *pte)
{
- queue_pte_unpin(__pa(pte));
+ // queue_pte_unpin(__pa(pte));
__make_page_writeable(pte);
free_page((unsigned long)pte);
}
#define PTE_TABLE_MASK ((PTRS_PER_PTE-1) * sizeof(pte_t))
#define PMD_TABLE_MASK ((PTRS_PER_PMD-1) * sizeof(pmd_t))
+#undef set_pte
+#define set_pte(pteptr, pteval) do { \
+ (*(pteptr) = pteval); \
+ /* printk("set_pte %p -> %08lx\n", pteptr, pteval); */ \
+} while (0)
+//void queue_l1_entry_update_queued(pte_t *ptr, unsigned long val);
+//#define set_pte(pteptr, pteval) queue_l1_entry_update_queued(pteptr, (pteval).pte_low)
+// #define ptep_get_and_clear(xp) __pte(xchg(&(xp)->pte_low, 0))
+//#undef pte_unmap
+//#define pte_unmap(pte) xen_flush_page_update_queue()
+#undef pmd_bad
+#define pmd_bad(x) (((x).pmd & (~PAGE_MASK & ~_PAGE_USER & ~_PAGE_PRESENT & ~0x800)) != (_KERNPG_TABLE & ~_PAGE_PRESENT))
/*
* copy one vm_area from one task to the other. Assumes the page tables
* already present in the new task to be cleared in the whole range
src_pgd = pgd_offset(src, address)-1;
dst_pgd = pgd_offset(dst, address)-1;
+ /* printk("copy_page_range src %p dst %p src_pgd %p dst_pgd %p %08lx-%08lx\n", */
+/* src, dst, src_pgd, dst_pgd, address, end); */
for (;;) {
pmd_t * src_pmd, * dst_pmd;
src_pmd = pmd_offset(src_pgd, address);
dst_pmd = pmd_alloc(dst, dst_pgd, address);
+ /* printk("src_pmd %p dst_pmd %p\n", src_pmd, dst_pmd); */
if (!dst_pmd)
goto nomem;
src_pte = pte_offset(src_pmd, address);
dst_pte = pte_alloc(dst, dst_pmd, address);
+ /* printk("src_pte %p(%p,%08lx,%08lx, %08lx) dst_pte %p\n", */
+/* src_pte, src_pmd, *src_pmd, pmd_page(*src_pmd), address, dst_pte); */
if (!dst_pte)
goto nomem;
if (pte_none(pte))
goto cont_copy_pte_range_noset;
if (!pte_present(pte)) {
+ printk("swap_dup call %p:%08lx\n",
+ src_pte, pte.pte_low);
swap_duplicate(pte_to_swp_entry(pte));
goto cont_copy_pte_range;
}
/* If it's a COW mapping, write protect it both in the parent and the child */
if (cow && pte_write(pte)) {
+ /* printk("ptep_set_wrprotect %p was %08lx\n", src_pte, *src_pte); */
+#if 0
/* XEN modification: modified ordering here to avoid RaW hazard. */
pte = *src_pte;
pte = pte_wrprotect(pte);
ptep_set_wrprotect(src_pte);
+#else
+ clear_bit(_PAGE_BIT_RW, src_pte); //ptep_set_wrprotect(src_pte);
+ pte = *src_pte;
+ /* printk("ptep_set_wrprotect %p now %08lx\n", src_pte, *src_pte); */
+#endif
}
/* If it's a shared mapping, mark it clean in the child */
out_unlock:
spin_unlock(&src->page_table_lock);
out:
+ /* printk("out\n"); */
return 0;
nomem:
return -ENOMEM;
}
+#undef set_pte
+#define set_pte(pteptr, pteval) queue_l1_entry_update(pteptr, (pteval).pte_low)
/*
* Return indicates whether a page was freed so caller can adjust rss
+extern unsigned long disconnected;
+extern void ptwr_reconnect(unsigned long);
+extern int writable_idx;
+extern void ptwr_flush(void);
/******************************************************************************
* arch/x86/memory.c
*
static void free_l2_table(struct pfn_info *page);
static void free_l1_table(struct pfn_info *page);
-static int mod_l2_entry(l2_pgentry_t *, l2_pgentry_t, unsigned long);
+int mod_l2_entry(l2_pgentry_t *, l2_pgentry_t, unsigned long);
static int mod_l1_entry(l1_pgentry_t *, l1_pgentry_t);
/* Used to defer flushing of memory structures. */
}
+static inline void set_l1_page_va(unsigned long pfn,
+ unsigned long va_idx)
+{
+ struct pfn_info *page;
+
+ page = &frame_table[pfn];
+ page->type_and_flags &= ~PGT_va_mask;
+ page->type_and_flags |= va_idx << PGT_va_shift;
+}
+
+
+#define NPRINTK if (0) printk
/* Update the L2 entry at pl2e to new value nl2e. pl2e is within frame pfn. */
-static int mod_l2_entry(l2_pgentry_t *pl2e,
+int mod_l2_entry(l2_pgentry_t *pl2e,
l2_pgentry_t nl2e,
unsigned long pfn)
{
return 0;
ol2e = mk_l2_pgentry(_ol2e);
+ NPRINTK("mod_l2_entry pl2e %p ol2e %08lx nl2e %08lx pfn %08lx\n",
+ pl2e, l2_pgentry_val(ol2e), l2_pgentry_val(nl2e), pfn);
if ( l2_pgentry_val(nl2e) & _PAGE_PRESENT )
{
/* Differ in mapping (bits 12-31) or presence (bit 0)? */
if ( unlikely(!get_page_from_l2e(nl2e, pfn)) )
return 0;
+ set_l1_page_va(l2_pgentry_val(nl2e) >> PAGE_SHIFT,
+ ((unsigned long)pl2e & (PAGE_SIZE-1)) >> 2);
+
if ( unlikely(!update_l2e(pl2e, ol2e, nl2e)) )
{
put_page_from_l2e(nl2e, pfn);
u32 x, y;
domid_t domid;
+ if (disconnected != ENTRIES_PER_L2_PAGETABLE)
+ ptwr_reconnect(0L);
+ if (writable_idx)
+ ptwr_flush();
+
switch ( cmd )
{
case MMUEXT_PIN_L1_TABLE:
perfc_incrc(calls_to_mmu_update);
perfc_addc(num_page_updates, count);
+ if (disconnected != ENTRIES_PER_L2_PAGETABLE)
+ ptwr_reconnect(0L);
+ if (writable_idx)
+ ptwr_flush();
+
for ( i = 0; i < count; i++ )
{
if ( unlikely(copy_from_user(&req, ureqs, sizeof(req)) != 0) )
perfc_incrc(calls_to_update_va);
+ if (disconnected != ENTRIES_PER_L2_PAGETABLE)
+ ptwr_reconnect(0L);
+ if (writable_idx)
+ ptwr_flush();
+
if ( unlikely(page_nr >= (HYPERVISOR_VIRT_START >> PAGE_SHIFT)) )
return -EINVAL;
for ( ; ; ) ;
}
+extern int mod_l2_entry(l2_pgentry_t *, l2_pgentry_t, unsigned long);
+unsigned long disconnected = ENTRIES_PER_L2_PAGETABLE;
+static unsigned long *writable_l1;
+#define NR_WRITABLES 4
+static unsigned long *writables[NR_WRITABLES];
+int writable_idx = 0;
+#define PRINTK if (0) printk
+#define NPRINTK if (0) printk
+
+void ptwr_reconnect(unsigned long addr)
+{
+ unsigned long pte;
+ unsigned long pfn;
+ struct pfn_info *page;
+ l2_pgentry_t *pl2e;
+ PRINTK("page fault in disconnected space: addr %08lx space %08lx\n",
+ addr, disconnected << L2_PAGETABLE_SHIFT);
+ pl2e = &linear_l2_table[disconnected];
+
+ if (__get_user(pte, writable_l1))
+ BUG();
+ pfn = pte >> PAGE_SHIFT;
+ page = &frame_table[pfn];
+
+ /* reconnect l1 page */
+ PRINTK(" pl2e %p l2e %08lx pfn %08lx taf %08x/%08x\n", pl2e,
+ l2_pgentry_val(*pl2e),
+ l1_pgentry_val(linear_pg_table[(unsigned long)pl2e >>
+ PAGE_SHIFT]) >> PAGE_SHIFT,
+ frame_table[l2_pgentry_to_pagenr(*pl2e)].type_and_flags,
+ frame_table[pfn].type_and_flags);
+ mod_l2_entry(pl2e, mk_l2_pgentry((l2_pgentry_val(*pl2e) & ~0x800) |
+ _PAGE_PRESENT),
+ l1_pgentry_val(linear_pg_table[(unsigned long)pl2e >>
+ PAGE_SHIFT]) >> PAGE_SHIFT);
+ PRINTK("now pl2e %p l2e %08lx taf %08x/%08x\n", pl2e,
+ l2_pgentry_val(*pl2e),
+ frame_table[l2_pgentry_to_pagenr(*pl2e)].type_and_flags,
+ frame_table[pfn].type_and_flags);
+ disconnected = ENTRIES_PER_L2_PAGETABLE;
+ /* make pt page write protected */
+ if (__get_user(pte, writable_l1))
+ BUG();
+ PRINTK("writable_l1 at %p is %08lx\n", writable_l1, pte);
+ pte &= ~_PAGE_RW;
+ if (__put_user(pte, writable_l1))
+ BUG();
+ PRINTK("writable_l1 at %p now %08lx\n", writable_l1, pte);
+ /* and try again */
+ return;
+}
+
+void ptwr_flush(void)
+{
+ unsigned long pte, pfn;
+ struct pfn_info *page;
+ int i;
+
+ for (i = 0; i < writable_idx; i++) {
+ if (__get_user(pte, writables[i]))
+ BUG();
+ pfn = pte >> PAGE_SHIFT;
+ page = &frame_table[pfn];
+ PRINTK("alloc l1 page %p\n", page);
+ if (!get_page_type(page, PGT_l1_page_table))
+ BUG();
+ /* make pt page writable */
+ PRINTK("writable_l1 at %p is %08lx\n", writables[i], pte);
+ pte &= ~_PAGE_RW;
+ if (__put_user(pte, writables[i]))
+ BUG();
+ PRINTK("writable_l1 at %p now %08lx\n", writables[i], pte);
+ }
+ writable_idx = 0;
+}
+
asmlinkage void do_page_fault(struct pt_regs *regs, long error_code)
{
struct guest_trap_bounce *gtb = guest_trap_bounce+smp_processor_id();
return; /* successfully copied the mapping */
}
+ if ((addr >> L2_PAGETABLE_SHIFT) == disconnected) {
+ ptwr_reconnect(addr);
+ return;
+ }
+
+ if (addr < PAGE_OFFSET && error_code & 2) {
+ /* write page fault, check if we're trying to modify an l1
+ page table */
+ unsigned long pte, pfn;
+ struct pfn_info *page;
+ l2_pgentry_t *pl2e;
+ NPRINTK("get user %p for va %08lx\n",
+ &linear_pg_table[addr>>PAGE_SHIFT], addr);
+ if (l2_pgentry_val(linear_l2_table[addr >> L2_PAGETABLE_SHIFT]) &
+ _PAGE_PRESENT &&
+ __get_user(pte, (unsigned long *)
+ &linear_pg_table[addr >> PAGE_SHIFT]) == 0) {
+ pfn = pte >> PAGE_SHIFT;
+ NPRINTK("check pte %08lx = pfn %08lx for va %08lx\n", pte, pfn, addr);
+ page = &frame_table[pfn];
+ if ((page->type_and_flags & PGT_type_mask) == PGT_l1_page_table) {
+ pl2e = &linear_l2_table[(page->type_and_flags &
+ PGT_va_mask) >> PGT_va_shift];
+ PRINTK("page_fault on l1 pt at va %08lx, pt for %08x, pfn %08lx\n",
+ addr, ((page->type_and_flags & PGT_va_mask) >>
+ PGT_va_shift) << L2_PAGETABLE_SHIFT, pfn);
+ if (l2_pgentry_val(*pl2e) >> PAGE_SHIFT != pfn) {
+ PRINTK("freeing l1 page %p\n", page);
+ if (writable_idx == NR_WRITABLES)
+ ptwr_flush();
+ writables[writable_idx++] = (unsigned long *)
+ &linear_pg_table[addr>>PAGE_SHIFT];
+ if ((page->type_and_flags & PGT_count_mask) != 1)
+ BUG();
+ put_page_type(page);
+ } else {
+ if (disconnected != ENTRIES_PER_L2_PAGETABLE)
+ ptwr_reconnect(addr);
+ PRINTK(" pl2e %p l2e %08lx pfn %08lx taf %08x/%08x\n",
+ pl2e, l2_pgentry_val(*pl2e),
+ l1_pgentry_val(linear_pg_table[(unsigned long)pl2e
+ >> PAGE_SHIFT]) >>
+ PAGE_SHIFT,
+ frame_table[l2_pgentry_to_pagenr(*pl2e)].
+ type_and_flags, frame_table[pfn].type_and_flags);
+ /* disconnect l1 page */
+ mod_l2_entry(pl2e, mk_l2_pgentry((l2_pgentry_val(*pl2e) &
+ ~_PAGE_PRESENT) | 0x800),
+ l1_pgentry_val(linear_pg_table
+ [(unsigned long)pl2e
+ >> PAGE_SHIFT]) >>
+ PAGE_SHIFT);
+ disconnected = (page->type_and_flags & PGT_va_mask) >>
+ PGT_va_shift;
+ PRINTK("now pl2e %p l2e %08lx taf %08x/%08x\n",
+ pl2e, l2_pgentry_val(*pl2e),
+ frame_table[l2_pgentry_to_pagenr(*pl2e)].
+ type_and_flags,
+ frame_table[pfn].type_and_flags);
+ writable_l1 = (unsigned long *)
+ &linear_pg_table[addr>>PAGE_SHIFT];
+ }
+ /* make pt page writable */
+ pte |= _PAGE_RW;
+ PRINTK("update %p pte to %08lx\n",
+ &linear_pg_table[addr>>PAGE_SHIFT], pte);
+ if (__put_user(pte, (unsigned long *)
+ &linear_pg_table[addr>>PAGE_SHIFT]))
+ BUG();
+ return;
+ }
+ }
+ }
+
if ( unlikely(p->mm.shadow_mode) &&
(addr < PAGE_OFFSET) && shadow_fault(addr, error_code) )
return; /* Returns TRUE if fault was handled. */
/* Has this page been validated for use as its current type? */
#define _PGT_validated 28
#define PGT_validated (1<<_PGT_validated)
- /* 28-bit count of uses of this frame as its current type. */
-#define PGT_count_mask ((1<<28)-1)
+ /* 10-bit most significant bits of va address if used as l1 page table */
+#define PGT_va_shift 18
+#define PGT_va_mask (((1<<10)-1)<<PGT_va_shift)
+ /* 18-bit count of uses of this frame as its current type. */
+#define PGT_count_mask ((1<<18)-1)
/* For safety, force a TLB flush when this page's type changes. */
#define _PGC_tlb_flush_on_type_change 31
projects / xen.git / commitdiff